{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import pandas as pd\n",
    "import matplotlib.pyplot as plt\n",
    "plt.style.use('classic')\n",
    "import mglearn"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# generate dataset\n",
    "X, y = mglearn.datasets.make_forge()\n",
    "# plot dataset\n",
    "mglearn.discrete_scatter(X[:, 0], X[:, 1],y)\n",
    "plt.legend([\"Class 0\", \"Class 1\"], loc=4)\n",
    "plt.xlabel(\"First feature\")\n",
    "plt.ylabel(\"Second feature\")\n",
    "print(\"X.shape: {}\".format(X.shape))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "X, y = mglearn.datasets.make_wave(n_samples=40)\n",
    "plt.plot(X, y, 'o')\n",
    "plt.ylim(-3, 3)\n",
    "plt.xlabel(\"Feature\")\n",
    "plt.ylabel(\"Target\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.datasets import load_breast_cancer\n",
    "cancer = load_breast_cancer()\n",
    "print(\"cancer.keys(): \\n{}\".format(cancer.keys()))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.datasets import load_boston\n",
    "boston = load_boston()\n",
    "print(\"Data shape: {}\".format(boston.data.shape))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "X, y = mglearn.datasets.load_extended_boston()\n",
    "print(\"X.shape: {}\".format(X.shape))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "mglearn.plots.plot_knn_classification(n_neighbors=5)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.model_selection import train_test_split\n",
    "X, y = mglearn.datasets.make_forge()\n",
    "X_train, X_test, y_train, y_test = train_test_split(X, y, random_state=0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.neighbors import KNeighborsClassifier\n",
    "clf = KNeighborsClassifier(n_neighbors=3)\n",
    "clf.fit(X_train, y_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "print(\"Test set predictions: {}\".format(clf.predict(X_test)))\n",
    "print(\"Test set accuracy: {:.2f}\".format(clf.score(X_test, y_test)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "fig, axes = plt.subplots(1, 3, figsize=(10, 3))\n",
    "for n_neighbors, ax in zip([1, 3, 9], axes):\n",
    "    # the fit method returns the object self, so we can instantiate\n",
    "    # and fit in one line\n",
    "    clf = KNeighborsClassifier(n_neighbors=n_neighbors).fit(X, y)\n",
    "    mglearn.plots.plot_2d_separator(clf, X, fill=True, eps=0.5, ax=ax, alpha=.4)\n",
    "    mglearn.discrete_scatter(X[:, 0], X[:, 1], y, ax=ax)\n",
    "    ax.set_title(\"{} neighbor(s)\".format(n_neighbors))\n",
    "    ax.set_xlabel(\"feature 0\")\n",
    "    ax.set_ylabel(\"feature 1\")\n",
    "axes[0].legend(loc=3)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.datasets import load_breast_cancer\n",
    "cancer = load_breast_cancer()\n",
    "X_train, X_test, y_train, y_test = train_test_split(cancer.data, cancer.target, stratify=cancer.target, random_state=66)\n",
    "training_accuracy = []\n",
    "test_accuracy = []\n",
    "# try n_neighbors from 1 to 10\n",
    "neighbors_settings = range(1, 11)\n",
    "for n_neighbors in neighbors_settings:\n",
    "    # build the model\n",
    "    clf = KNeighborsClassifier(n_neighbors=n_neighbors)\n",
    "    clf.fit(X_train, y_train)\n",
    "    # record training set accuracy\n",
    "    training_accuracy.append(clf.score(X_train, y_train))\n",
    "    # record generalization accuracy\n",
    "    test_accuracy.append(clf.score(X_test, y_test))\n",
    "plt.plot(neighbors_settings, training_accuracy, label=\"training accuracy\")\n",
    "plt.plot(neighbors_settings, test_accuracy, label=\"test accuracy\")\n",
    "plt.ylabel(\"Accuracy\")\n",
    "plt.xlabel(\"n_neighbors\")\n",
    "plt.legend()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "mglearn.plots.plot_knn_regression(n_neighbors=6)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.neighbors import KNeighborsRegressor\n",
    "X, y = mglearn.datasets.make_wave(n_samples=40)\n",
    "# split the wave dataset into a training and a test set\n",
    "X_train, X_test, y_train, y_test = train_test_split(X, y, random_state=0)\n",
    "# instantiate the model and set the number of neighbors to consider to 3\n",
    "reg = KNeighborsRegressor(n_neighbors=3)\n",
    "# fit the model using the training data and training targets\n",
    "reg.fit(X_train, y_train)\n",
    "print(\"Test set predictions:\\n{}\".format(reg.predict(X_test)))\n",
    "print(\"Test set R^2: {:.2f}\".format(reg.score(X_test, y_test)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "fig, axes = plt.subplots(1, 3, figsize=(15, 4))\n",
    "# create 1,000 data points, evenly spaced between -3 and 3\n",
    "line = np.linspace(-3, 3, 1000).reshape(-1, 1)\n",
    "for n_neighbors, ax in zip([1, 3, 9], axes):\n",
    "    # make predictions using 1, 3, or 9 neighbors\n",
    "    reg = KNeighborsRegressor(n_neighbors=n_neighbors)\n",
    "    reg.fit(X_train, y_train)\n",
    "    ax.plot(line, reg.predict(line))\n",
    "    ax.plot(X_train, y_train, '^', c=mglearn.cm2(0), markersize=8)\n",
    "    ax.plot(X_test, y_test, 'v', c=mglearn.cm2(1), markersize=8)\n",
    "    ax.set_title(\"{} neighbor(s)\\n train score: {:.2f} test score: {:.2f}\".format(n_neighbors, reg.score(X_train, y_train),reg.score(X_test, y_test)))\n",
    "    ax.set_xlabel(\"Feature\")\n",
    "    ax.set_ylabel(\"Target\")\n",
    "axes[0].legend([\"Model predictions\", \"Training data/target\", \"Test data/target\"], loc=\"best\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "mglearn.plots.plot_linear_regression_wave()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.linear_model import LinearRegression\n",
    "X, y = mglearn.datasets.make_wave(n_samples=60)\n",
    "X_train, X_test, y_train, y_test = train_test_split(X, y, random_state=42)\n",
    "lr = LinearRegression().fit(X_train, y_train)\n",
    "print(\"lr.coef_: {}\".format(lr.coef_))\n",
    "print(\"lr.intercept_: {}\".format(lr.intercept_))\n",
    "print(\"Training set score: {:.2f}\".format(lr.score(X_train, y_train)))\n",
    "print(\"Test set score: {:.2f}\".format(lr.score(X_test, y_test)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "X, y = mglearn.datasets.load_extended_boston()\n",
    "X_train, X_test, y_train, y_test = train_test_split(X, y, random_state=0)\n",
    "lr = LinearRegression().fit(X_train, y_train)\n",
    "print(\"Training set score: {:.2f}\".format(lr.score(X_train, y_train)))\n",
    "print(\"Test set score: {:.2f}\".format(lr.score(X_test, y_test)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.linear_model import Ridge\n",
    "ridge = Ridge().fit(X_train, y_train)\n",
    "print(\"Training set score: {:.2f}\".format(ridge.score(X_train, y_train)))\n",
    "print(\"Test set score: {:.2f}\".format(ridge.score(X_test, y_test)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "ridge10 = Ridge(alpha=10).fit(X_train, y_train)\n",
    "print(\"Training set score: {:.2f}\".format(ridge10.score(X_train, y_train)))\n",
    "print(\"Test set score: {:.2f}\".format(ridge10.score(X_test, y_test)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "ridge01 = Ridge(alpha=0.1).fit(X_train, y_train)\n",
    "print(\"Training set score: {:.2f}\".format(ridge01.score(X_train, y_train)))\n",
    "print(\"Test set score: {:.2f}\".format(ridge01.score(X_test, y_test)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "plt.figure(figsize=(12.8,9.6))\n",
    "plt.plot(ridge.coef_, 's', label=\"Ridge alpha=1\")\n",
    "plt.plot(ridge10.coef_, '^', label=\"Ridge alpha=10\")\n",
    "plt.plot(ridge01.coef_, 'v', label=\"Ridge alpha=0.1\")\n",
    "plt.plot(lr.coef_, 'o', label=\"LinearRegression\")\n",
    "plt.xlabel(\"Coefficient index\")\n",
    "plt.ylabel(\"Coefficient magnitude\")\n",
    "plt.hlines(0, 0, len(lr.coef_))\n",
    "plt.ylim(-25, 25)\n",
    "plt.legend()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "mglearn.plots.plot_ridge_n_samples()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.linear_model import Lasso\n",
    "lasso = Lasso().fit(X_train, y_train)\n",
    "print(\"Training set score: {:.2f}\".format(lasso.score(X_train, y_train)))\n",
    "print(\"Test set score: {:.2f}\".format(lasso.score(X_test, y_test)))\n",
    "print(\"Number of features used: {}\".format(np.sum(lasso.coef_ != 0)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# we increase the default setting of \"max_iter\",\n",
    "# otherwise the model would warn us that we should increase max_iter.\n",
    "lasso001 = Lasso(alpha=0.01, max_iter=100000).fit(X_train, y_train)\n",
    "print(\"Training set score: {:.2f}\".format(lasso001.score(X_train, y_train)))\n",
    "print(\"Test set score: {:.2f}\".format(lasso001.score(X_test, y_test)))\n",
    "print(\"Number of features used: {}\".format(np.sum(lasso001.coef_ != 0)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "lasso00001 = Lasso(alpha=0.0001, max_iter=100000).fit(X_train, y_train)\n",
    "print(\"Training set score: {:.2f}\".format(lasso00001.score(X_train, y_train)))\n",
    "print(\"Test set score: {:.2f}\".format(lasso00001.score(X_test, y_test)))\n",
    "print(\"Number of features used: {}\".format(np.sum(lasso00001.coef_ != 0)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "plt.figure(figsize=(12.8,9.6))\n",
    "plt.plot(lasso.coef_, 's', label=\"Lasso alpha=1\")\n",
    "plt.plot(lasso001.coef_, '^', label=\"Lasso alpha=0.01\")\n",
    "plt.plot(lasso00001.coef_, 'v', label=\"Lasso alpha=0.0001\")\n",
    "plt.plot(ridge01.coef_, 'o', label=\"Ridge alpha=0.1\")\n",
    "plt.legend(ncol=2, loc=(0, 1.05))\n",
    "plt.ylim(-25, 25)\n",
    "plt.xlabel(\"Coefficient index\")\n",
    "plt.ylabel(\"Coefficient magnitude\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.linear_model import LogisticRegression\n",
    "from sklearn.svm import LinearSVC\n",
    "\n",
    "X,y = mglearn.datasets.make_forge()\n",
    "fig, axes = plt.subplots(1,2,figsize=(10,3))\n",
    "for model,ax in zip([LinearSVC(),LogisticRegression()],axes):\n",
    "    clf = model.fit(X,y)\n",
    "    mglearn.plots.plot_2d_separator(clf,X,fill=False,eps=0.5, ax=ax,alpha=0.7)\n",
    "    mglearn.discrete_scatter(X[:,0],X[:,1],y,ax=ax)\n",
    "    ax.set_title(\"{}\".format(clf.__class__.__name__))\n",
    "    ax.set_xlabel('Feature 0')\n",
    "    ax.set_ylabel(\"Feature 1\")\n",
    "axes[0].legend()    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "model.coef_"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "mglearn.plots.plot_linear_svc_regularization()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.datasets import load_breast_cancer\n",
    "cancer = load_breast_cancer()\n",
    "X_train, X_test, y_train, y_test = train_test_split(cancer.data, cancer.target, stratify=cancer.target, random_state=42)\n",
    "logreg = LogisticRegression(max_iter=10000).fit(X_train, y_train)\n",
    "print(\"Training set score: {:.3f}\".format(logreg.score(X_train, y_train)))\n",
    "print(\"Test set score: {:.3f}\".format(logreg.score(X_test, y_test)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "logreg100 = LogisticRegression(C=100,max_iter=10000).fit(X_train, y_train)\n",
    "print(\"Training set score: {:.3f}\".format(logreg100.score(X_train, y_train)))\n",
    "print(\"Test set score: {:.3f}\".format(logreg100.score(X_test, y_test)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "logreg001 = LogisticRegression(C=0.01,max_iter=10000).fit(X_train, y_train)\n",
    "print(\"Training set score: {:.3f}\".format(logreg001.score(X_train, y_train)))\n",
    "print(\"Test set score: {:.3f}\".format(logreg001.score(X_test, y_test)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "plt.plot(logreg.coef_.T, 'o', label=\"C=1\")\n",
    "plt.plot(logreg100.coef_.T, '^', label=\"C=100\")\n",
    "plt.plot(logreg001.coef_.T, 'v', label=\"C=0.001\")\n",
    "plt.xticks(range(cancer.data.shape[1]), cancer.feature_names, rotation=90)\n",
    "plt.hlines(0, 0, cancer.data.shape[1])\n",
    "plt.ylim(-5, 5)\n",
    "plt.xlabel(\"Coefficient index\")\n",
    "plt.ylabel(\"Coefficient magnitude\")\n",
    "plt.legend()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "for C, marker in zip([0.001,1,100], ['v', 'o', '^']):\n",
    "    lr_l1 = LogisticRegression(C=C, penalty='l1',max_iter=10000,solver='liblinear').fit(X_train,y_train)\n",
    "    print('\"Training accuracy of l1 logreg with C={:.3f}: {:.2f}'.format(C,lr_l1.score(X_train, y_train)))\n",
    "    print(\"Test accuracy of l1 logreg with C={:.3f}: {:.2f}\".format(C, lr_l1.score(X_test, y_test)))\n",
    "    plt.plot(lr_l1.coef_.T, marker, label=\"C={:.3f}\".format(C))\n",
    "plt.xticks(range(cancer.data.shape[1]), cancer.feature_names, rotation=90)\n",
    "plt.hlines(0, 0, cancer.data.shape[1])\n",
    "plt.xlabel(\"Coefficient index\")\n",
    "plt.ylabel(\"Coefficient magnitude\")\n",
    "plt.ylim(-5, 5)\n",
    "plt.legend(loc=3)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.datasets import make_blobs\n",
    "X, y = make_blobs(random_state=42)\n",
    "mglearn.discrete_scatter(X[:, 0], X[:, 1], y)\n",
    "plt.xlabel(\"Feature 0\")\n",
    "plt.ylabel(\"Feature 1\")\n",
    "plt.legend([\"Class 0\", \"Class 1\", \"Class 2\"])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "linear_svm = LinearSVC().fit(X, y)\n",
    "print(\"Coefficient shape: \", linear_svm.coef_.shape)\n",
    "print(\"Intercept shape: \", linear_svm.intercept_.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "mglearn.discrete_scatter(X[:, 0], X[:, 1], y)\n",
    "line = np.linspace(-15, 15)\n",
    "for coef, intercept, color in zip(linear_svm.coef_, linear_svm.intercept_, ['b', 'r', 'g']):\n",
    "    plt.plot(line, -(line * coef[0] + intercept) / coef[1], c=color)\n",
    "plt.ylim(-10, 15)\n",
    "plt.xlim(-10, 8)\n",
    "plt.xlabel(\"Feature 0\")\n",
    "plt.ylabel(\"Feature 1\")\n",
    "plt.legend(['Class 0', 'Class 1', 'Class 2', 'Line class 0', 'Line class 1', 'Line class 2'], loc=(1.01, 0.3))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "mglearn.plots.plot_2d_classification(linear_svm, X, fill=True, alpha=.7)\n",
    "mglearn.discrete_scatter(X[:, 0], X[:, 1], y)\n",
    "line = np.linspace(-15, 15)\n",
    "for coef, intercept, color in zip(linear_svm.coef_, linear_svm.intercept_, ['b', 'r', 'g']):\n",
    "    plt.plot(line, -(line * coef[0] + intercept) / coef[1], c=color)\n",
    "plt.legend(['Class 0', 'Class 1', 'Class 2', 'Line class 0', 'Line class 1', 'Line class 2'], loc=(1.01, 0.3))\n",
    "plt.xlabel(\"Feature 0\")\n",
    "plt.ylabel(\"Feature 1\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "X = np.array([[0, 1, 0, 1],\n",
    "              [1, 0, 1, 1],\n",
    "              [0, 0, 0, 1],\n",
    "              [1, 0, 1, 0]])\n",
    "y = np.array([0, 1, 0, 1])\n",
    "counts = {}\n",
    "for label in np.unique(y):\n",
    "    counts[label] = X[y==label].sum(axis=0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "mglearn.plots.plot_animal_tree()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.tree import DecisionTreeClassifier\n",
    "\n",
    "cancer = load_breast_cancer()\n",
    "X_train,X_test,y_train,y_test = train_test_split(cancer.data, cancer.target, stratify=cancer.target, random_state=1026)\n",
    "tree = DecisionTreeClassifier(random_state=1026)\n",
    "tree.fit(X_train,y_train)\n",
    "print(\"Accuracy on training set: {:.3f}\".format(tree.score(X_train, y_train)))\n",
    "print(\"Accuracy on test set: {:.3f}\".format(tree.score(X_test, y_test)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "tree = DecisionTreeClassifier(max_depth=4, random_state=1026)\n",
    "tree.fit(X_train,y_train)\n",
    "print(\"Accuracy on training set: {:.3f}\".format(tree.score(X_train, y_train)))\n",
    "print(\"Accuracy on test set: {:.3f}\".format(tree.score(X_test, y_test)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.tree import export_graphviz\n",
    "export_graphviz(tree, out_file=\"tree.dot\", class_names=[\"malignant\", \"benign\"], feature_names=cancer.feature_names, impurity=False, filled=True)\n",
    "import graphviz\n",
    "with open('tree.dot') as f:\n",
    "    dot_graph = f.read()\n",
    "graphviz.Source(dot_graph)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def plot_feature_importances_cancer(model):\n",
    "    n_features = cancer.data.shape[1]\n",
    "    plt.barh(range(n_features), model.feature_importances_, align='center')\n",
    "    plt.yticks(np.arange(n_features), cancer.feature_names)\n",
    "    plt.xlabel(\"Feature importance\")\n",
    "    plt.ylabel(\"Feature\")\n",
    "plot_feature_importances_cancer(tree)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "tree = mglearn.plots.plot_tree_not_monotone()\n",
    "display(tree)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.ensemble import RandomForestClassifier\n",
    "from sklearn.datasets import make_moons\n",
    "X, y = make_moons(n_samples=100, noise=0.25, random_state=3)\n",
    "X_train, X_test, y_train, y_test = train_test_split(X, y, stratify=y,\n",
    " random_state=42)\n",
    "forest = RandomForestClassifier(n_estimators=5, random_state=2)\n",
    "forest.fit(X_train, y_train)\n",
    "print(\"Accuracy on training set: {:.3f}\".format(forest.score(X_train, y_train)))\n",
    "print(\"Accuracy on test set: {:.3f}\".format(forest.score(X_test, y_test)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "fig, axes = plt.subplots(2, 3, figsize=(20, 10))\n",
    "for i, (ax, tree) in enumerate(zip(axes.ravel(), forest.estimators_)):\n",
    " ax.set_title(\"Tree {}\".format(i))\n",
    " mglearn.plots.plot_tree_partition(X_train, y_train, tree, ax=ax)\n",
    "mglearn.plots.plot_2d_separator(forest, X_train, fill=True, ax=axes[-1, -1],\n",
    " alpha=.4)\n",
    "axes[-1, -1].set_title(\"Random Forest\")\n",
    "mglearn.discrete_scatter(X_train[:, 0], X_train[:, 1], y_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "X_train, X_test, y_train, y_test = train_test_split(cancer.data, cancer.target, random_state=1026)\n",
    "forest = RandomForestClassifier(n_estimators=100, random_state=1026)\n",
    "forest.fit(X_train, y_train)\n",
    "print(\"Accuracy on training set: {:.3f}\".format(forest.score(X_train, y_train)))\n",
    "print(\"Accuracy on test set: {:.3f}\".format(forest.score(X_test, y_test)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "plot_feature_importances_cancer(forest)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.ensemble import GradientBoostingClassifier\n",
    "X_train, X_test, y_train, y_test = train_test_split(cancer.data, cancer.target, random_state=1026)\n",
    "gbrt = GradientBoostingClassifier(random_state=1026)\n",
    "gbrt.fit(X_train,y_train)\n",
    "print(\"Accuracy on training set: {:.3f}\".format(gbrt.score(X_train, y_train)))\n",
    "print(\"Accuracy on test set: {:.3f}\".format(gbrt.score(X_test, y_test)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "gbrt = GradientBoostingClassifier(random_state=1026,max_depth=1)\n",
    "gbrt.fit(X_train,y_train)\n",
    "print(\"Accuracy on training set: {:.3f}\".format(gbrt.score(X_train, y_train)))\n",
    "print(\"Accuracy on test set: {:.3f}\".format(gbrt.score(X_test, y_test)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "gbrt = GradientBoostingClassifier(random_state=1026, learning_rate=0.01)\n",
    "gbrt.fit(X_train, y_train)\n",
    "print(\"Accuracy on training set: {:.3f}\".format(gbrt.score(X_train, y_train)))\n",
    "print(\"Accuracy on test set: {:.3f}\".format(gbrt.score(X_test, y_test)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "gbrt = GradientBoostingClassifier(random_state=1026, max_depth=1)\n",
    "gbrt.fit(X_train, y_train)\n",
    "plot_feature_importances_cancer(gbrt)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "X,y = make_blobs(centers=4, random_state=8)\n",
    "y = y%2\n",
    "mglearn.discrete_scatter(X[:,0],X[:,1],y)\n",
    "plt.xlabel('Feature 0')\n",
    "plt.xlabel('Feature 1')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.svm import LinearSVC\n",
    "linear_svm = LinearSVC().fit(X,y)\n",
    "mglearn.plots.plot_2d_separator(linear_svm, X)\n",
    "mglearn.discrete_scatter(X[:,0],X[:,1],y)\n",
    "plt.xlabel('Feature 0')\n",
    "plt.xlabel('Feature 1')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "X_new = np.hstack([X, X[:, 1:] ** 2])\n",
    "from mpl_toolkits.mplot3d import Axes3D, axes3d\n",
    "figure = plt.figure()\n",
    "# visualize in 3D\n",
    "ax = Axes3D(figure, elev=-152, azim=-26)\n",
    "mask = y == 0\n",
    "ax.scatter(X_new[mask, 0], X_new[mask, 1], X_new[mask, 2], c='b', cmap=mglearn.cm2, s=60)\n",
    "ax.scatter(X_new[~mask, 0], X_new[~mask, 1], X_new[~mask, 2], c='r', marker='^', cmap=mglearn.cm2, s=60)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "linear_svm_3d = LinearSVC().fit(X_new, y)\n",
    "coef, intercept = linear_svm_3d.coef_.ravel(), linear_svm_3d.intercept_\n",
    "# show linear decision boundary\n",
    "figure = plt.figure()\n",
    "ax = Axes3D(figure, elev=-152, azim=-26)\n",
    "xx = np.linspace(X_new[:, 0].min() - 2, X_new[:, 0].max() + 2, 50)\n",
    "yy = np.linspace(X_new[:, 1].min() - 2, X_new[:, 1].max() + 2, 50)\n",
    "XX, YY = np.meshgrid(xx, yy)\n",
    "ZZ = (coef[0] * XX + coef[1] * YY + intercept) / -coef[2]\n",
    "ax.plot_surface(XX, YY, ZZ, rstride=8, cstride=8, alpha=0.3)\n",
    "ax.scatter(X_new[mask, 0], X_new[mask, 1], X_new[mask, 2], c='b',\n",
    " cmap=mglearn.cm2, s=60)\n",
    "ax.scatter(X_new[~mask, 0], X_new[~mask, 1], X_new[~mask, 2], c='r', marker='^',\n",
    " cmap=mglearn.cm2, s=60)\n",
    "ax.set_xlabel(\"feature0\")\n",
    "ax.set_ylabel(\"feature1\")\n",
    "ax.set_zlabel(\"feature0 ** 2\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "ZZ = YY ** 2\n",
    "dec = linear_svm_3d.decision_function(np.c_[XX.ravel(), YY.ravel(), ZZ.ravel()])\n",
    "plt.contourf(XX, YY, dec.reshape(XX.shape), levels=[dec.min(), 0, dec.max()],\n",
    " cmap=mglearn.cm2, alpha=0.5)\n",
    "mglearn.discrete_scatter(X[:, 0], X[:, 1], y)\n",
    "plt.xlabel(\"Feature 0\")\n",
    "plt.ylabel(\"Feature 1\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.svm import SVC\n",
    "X,y = mglearn.tools.make_handcrafted_dataset()\n",
    "svm = SVC(kernel='rbf', C=10, gamma=0.1).fit(X,y)\n",
    "mglearn.plots.plot_2d_separator(svm, X, eps=0.5)\n",
    "mglearn.discrete_scatter(X[:,0], X[:,1], y)\n",
    "sv = svm.support_vectors_\n",
    "sv_labels = svm.dual_coef_.ravel() > 0\n",
    "mglearn.discrete_scatter(sv[:,0], sv[:,1], sv_labels, s=15, markeredgewidth=3)\n",
    "plt.xlabel(\"Feature 0\")\n",
    "plt.ylabel(\"Feature 1\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "fig, axes = plt.subplots(3, 3, figsize=(15, 10))\n",
    "for ax, C in zip(axes, [-1, 0, 3]):\n",
    "    for a, gamma in zip(ax, range(-1, 2)):\n",
    "        mglearn.plots.plot_svm(log_C=C, log_gamma=gamma, ax=a)\n",
    "\n",
    "axes[0, 0].legend([\"class 0\", \"class 1\", \"sv class 0\", \"sv class 1\"], ncol=4, loc=(.9, 1.2))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "X_train, X_test, y_train, y_test = train_test_split(cancer.data, cancer.target, random_state=1026)\n",
    "svc = SVC(C=1000)\n",
    "svc.fit(X_train,y_train)\n",
    "print(\"Accuracy on training set: {:.2f}\".format(svc.score(X_train, y_train)))\n",
    "print(\"Accuracy on test set: {:.2f}\".format(svc.score(X_test, y_test)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "plt.plot(X_train.min(axis=0), 'o', label=\"min\")\n",
    "plt.plot(X_train.max(axis=0), '^', label=\"max\")\n",
    "plt.legend(loc=4)\n",
    "plt.xlabel(\"Feature index\")\n",
    "plt.ylabel(\"Feature magnitude\")\n",
    "plt.yscale(\"log\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# compute the minimum value per feature on the training set\n",
    "min_on_training = X_train.min(axis=0)\n",
    "# compute the range of each feature (max - min) on the training set\n",
    "range_on_training = (X_train - min_on_training).max(axis=0)\n",
    "# subtract the min, and divide by range\n",
    "# afterward, min=0 and max=1 for each feature\n",
    "X_train_scaled = (X_train - min_on_training) / range_on_training\n",
    "print(\"Minimum for each feature\\n{}\".format(X_train_scaled.min(axis=0)))\n",
    "print(\"Maximum for each feature\\n {}\".format(X_train_scaled.max(axis=0)))\n",
    "# use THE SAME transformation on the test set,\n",
    "# using min and range of the training set (see Chapter 3 for details)\n",
    "X_test_scaled = (X_test - min_on_training) / range_on_training\n",
    "svc = SVC()\n",
    "svc.fit(X_train_scaled, y_train)\n",
    "print(\"Accuracy on training set: {:.3f}\".format(svc.score(X_train_scaled, y_train)))\n",
    "print(\"Accuracy on test set: {:.3f}\".format(svc.score(X_test_scaled, y_test)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "line = np.linspace(-3,3,1000)\n",
    "plt.plot(line, np.tanh(line), label='tanh')\n",
    "plt.plot(line, np.maximum(line, 0), label='relu')\n",
    "plt.legend(loc='best')\n",
    "plt.xlabel('x')\n",
    "plt.ylabel('relu(x),tanh(x)')\n",
    "mglearn.plots.plot_two_hidden_layer_graph()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.neural_network import MLPClassifier\n",
    "from sklearn.datasets import make_moons\n",
    "\n",
    "X,y = make_moons(n_samples=100, noise=0.25, random_state=3)\n",
    "X_train, X_test, y_train, y_test = train_test_split(X, y, stratify=y, random_state=1026)\n",
    "mlp = MLPClassifier(solver='lbfgs', random_state=1026).fit(X_train, y_train)\n",
    "mglearn.plots.plot_2d_separator(mlp, X_train, fill=True, alpha=0.3)\n",
    "mglearn.discrete_scatter(X_train[:, 0], X_train[:, 1], y_train)\n",
    "plt.xlabel(\"Feature 0\")\n",
    "plt.ylabel(\"Feature 1\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "mlp = MLPClassifier(solver='lbfgs', random_state=1026, hidden_layer_sizes=[10])\n",
    "mlp.fit(X_train, y_train)\n",
    "mglearn.plots.plot_2d_separator(mlp, X_train, fill=True, alpha=.3)\n",
    "mglearn.discrete_scatter(X_train[:, 0], X_train[:, 1], y_train)\n",
    "plt.xlabel(\"Feature 0\")\n",
    "plt.ylabel(\"Feature 1\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# using two hidden layers, with 10 units each\n",
    "mlp = MLPClassifier(solver='lbfgs', max_iter=350, random_state=1026, hidden_layer_sizes=[10, 10])\n",
    "mlp.fit(X_train, y_train)\n",
    "mglearn.plots.plot_2d_separator(mlp, X_train, fill=True, alpha=.3)\n",
    "mglearn.discrete_scatter(X_train[:, 0], X_train[:, 1], y_train)\n",
    "plt.xlabel(\"Feature 0\")\n",
    "plt.ylabel(\"Feature 1\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# using two hidden layers, with 10 units each, now with tanh nonlinearity\n",
    "mlp = MLPClassifier(solver='lbfgs', max_iter=350, activation='tanh', random_state=1026, hidden_layer_sizes=[10, 10])\n",
    "mlp.fit(X_train, y_train)\n",
    "mglearn.plots.plot_2d_separator(mlp, X_train, fill=True, alpha=.3)\n",
    "mglearn.discrete_scatter(X_train[:, 0], X_train[:, 1], y_train)\n",
    "plt.xlabel(\"Feature 0\")\n",
    "plt.ylabel(\"Feature 1\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "fig, axes = plt.subplots(2, 4, figsize=(20, 8))\n",
    "for axx, n_hidden_nodes in zip(axes, [10, 100]):\n",
    "    for ax, alpha in zip(axx, [0.0001, 0.01, 0.1, 1]):\n",
    "        mlp = MLPClassifier(solver='lbfgs', max_iter=800, random_state=1026, hidden_layer_sizes=[n_hidden_nodes, n_hidden_nodes], alpha=alpha)\n",
    "        mlp.fit(X_train, y_train)\n",
    "        mglearn.plots.plot_2d_separator(mlp, X_train, fill=True, alpha=.3, ax=ax)\n",
    "        mglearn.discrete_scatter(X_train[:, 0], X_train[:, 1], y_train, ax=ax)\n",
    "        ax.set_title(\"n_hidden=[{}, {}]\\nalpha={:.4f}\".format(n_hidden_nodes, n_hidden_nodes, alpha))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "fig, axes = plt.subplots(2, 4, figsize=(20, 8))\n",
    "for i, ax in enumerate(axes.ravel()):\n",
    "    mlp = MLPClassifier(solver='lbfgs', random_state=i, hidden_layer_sizes=[100, 100])\n",
    "    mlp.fit(X_train, y_train)\n",
    "    mglearn.plots.plot_2d_separator(mlp, X_train, fill=True, alpha=.3, ax=ax)\n",
    "    mglearn.discrete_scatter(X_train[:, 0], X_train[:, 1], y_train, ax=ax)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "X_train, X_test, y_train, y_test = train_test_split(cancer.data, cancer.target, random_state=0)\n",
    "mlp = MLPClassifier(random_state=1026)\n",
    "mlp.fit(X_train, y_train)\n",
    "print(\"Accuracy on training set: {:.3f}\".format(mlp.score(X_train, y_train)))\n",
    "print(\"Accuracy on test set: {:.3f}\".format(mlp.score(X_test, y_test)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "mean_on_train = X_train.mean(axis=0)\n",
    "std_on_train = X_train.std(axis=0)\n",
    "X_train_scaled = (X_train - mean_on_train)/std_on_train\n",
    "X_test_scaled = (X_test- mean_on_train)/std_on_train\n",
    "mlp = MLPClassifier(random_state=1026,max_iter=500,alpha=1)\n",
    "mlp.fit(X_train_scaled,y_train)\n",
    "print(\"Accuracy on training set: {:.3f}\".format(mlp.score(X_train_scaled, y_train)))\n",
    "print(\"Accuracy on test set: {:.3f}\".format(mlp.score(X_test_scaled, y_test)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "plt.figure(figsize=(20, 5))\n",
    "plt.imshow(mlp.coefs_[0], interpolation='none', cmap='viridis')\n",
    "plt.yticks(range(30), cancer.feature_names)\n",
    "plt.xlabel(\"Columns in weight matrix\")\n",
    "plt.ylabel(\"Input feature\")\n",
    "plt.colorbar()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.ensemble import GradientBoostingClassifier\n",
    "from sklearn.datasets import make_blobs, make_circles\n",
    "X, y = make_circles(noise=0.25, factor=0.5, random_state=1026)\n",
    "y_named = np.array(['blue','red'])[y]\n",
    "X_train, X_test, y_train_named, y_test_named, y_train, y_test = train_test_split(X, y_named, y, random_state=1026)\n",
    "# build the gradient boosting model\n",
    "gbrt = GradientBoostingClassifier(random_state=1026)\n",
    "gbrt.fit(X_train, y_train_named)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "print(\"X_test.shape: {}\".format(X_test.shape))\n",
    "print(\"Decision function shape: {}\".format(gbrt.decision_function(X_test).shape))\n",
    "print(\"Decision function:{}\".format(gbrt.decision_function(X_test)[:6]))\n",
    "print(\"Thresholded decision function:\\n{}\".format(gbrt.decision_function(X_test) > 0))\n",
    "print(\"Predictions:{}\".format(gbrt.predict(X_test)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# make the boolean True/False into 0 and 1\n",
    "greater_zero = (gbrt.decision_function(X_test) > 0).astype(int)\n",
    "# use 0 and 1 as indices into classes_\n",
    "pred = gbrt.classes_[greater_zero]\n",
    "# pred is the same as the output of gbrt.predict\n",
    "print(\"pred is equal to predictions: {}\".format(np.all(pred == gbrt.predict(X_test))))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "decision_function = gbrt.decision_function(X_test)\n",
    "print(\"Decision function minimum: {:.2f} maximum: {:.2f}\".format(np.min(decision_function), np.max(decision_function)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "fig, axes = plt.subplots(1, 2, figsize=(13, 5))\n",
    "mglearn.tools.plot_2d_separator(gbrt, X, ax=axes[0], alpha=.4, fill=True, cm=mglearn.cm2)\n",
    "scores_image = mglearn.tools.plot_2d_scores(gbrt, X, ax=axes[1], alpha=.5, cm=mglearn.ReBl, function='predict_proba')\n",
    "for ax in axes:\n",
    " # plot training and test points\n",
    " mglearn.discrete_scatter(X_test[:, 0], X_test[:, 1], y_test, markers='^', ax=ax)\n",
    " mglearn.discrete_scatter(X_train[:, 0], X_train[:, 1], y_train, markers='o', ax=ax)\n",
    " ax.set_xlabel(\"Feature 0\")\n",
    " ax.set_ylabel(\"Feature 1\")\n",
    "cbar = plt.colorbar(scores_image, ax=axes.tolist())\n",
    "axes[0].legend([\"Test class 0\", \"Test class 1\", \"Train class 0\", \"Train class 1\"], ncol=4, loc=(.1, 1.1))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "fig, axes = plt.subplots(1, 2, figsize=(13, 5))\n",
    "mglearn.tools.plot_2d_separator(gbrt, X, ax=axes[0], alpha=.4, fill=True, cm=mglearn.cm2)\n",
    "scores_image = mglearn.tools.plot_2d_scores(gbrt, X, ax=axes[1], alpha=.5, cm=mglearn.ReBl, function='predict_proba')\n",
    "for ax in axes:\n",
    "    # plot training and test points\n",
    "    mglearn.discrete_scatter(X_test[:, 0], X_test[:, 1], y_test,    markers='^', ax=ax)\n",
    "    # mglearn.discrete_scatter(X_train[:, 0], X_train[:, 1], y_train,    markers='o', ax=ax)\n",
    "    ax.set_xlabel(\"Feature 0\")\n",
    "    ax.set_ylabel(\"Feature 1\")\n",
    "cbar = plt.colorbar(scores_image, ax=axes.tolist())\n",
    "axes[0].legend([\"Test class 0\", \"Test class 1\", \"Train class 0\", \"Train class 1\"], ncol=4, loc=(.1, 1.1))"
   ]
  }
 ],
 "metadata": {
  "interpreter": {
   "hash": "e21935a792cabaadabb39e077efde3aadff814271e124eae6fb74a8aae0f5081"
  },
  "kernelspec": {
   "display_name": "Python 3.10.5 64-bit",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.5"
  },
  "orig_nbformat": 4
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
